import openpyxl
import requests
from bs4 import BeautifulSoup

DOWNLOAD_URL = 'https://journals.sagepub.com/toc/stja/20/3'

allResult = []
Result = ""
#Output Case: Nicholas J. Cox, 2020, Speaking Stata: More ways for rowwise, Stata Joural, 20(2): 481–488. [-PDF1-](https://sci-hub.ren/10.1177/1536867X20931007), [-PDF2-](https://journals.sagepub.com/doi/pdf/10.1177/1536867X20931007)

paper_url_list = []

def parse_html(html):
    soup = BeautifulSoup(html,"lxml")

    paper_list_soup = soup.find_all('table',attrs={'class': 'articleEntry'})

    for paper in paper_list_soup:

        name = paper.find('h3',attrs={'class':'heading-title'}).find('span').getText()
        author_link = paper.find_all('span',attrs={'class':'contribDegrees'})
        allAuthor = ''
        for author_entryAuthor in author_link:
            author = author_entryAuthor.find('a',attrs={'class':'entryAuthor'})
            allAuthor = allAuthor + author.getText() if author else ' --,'
            allAuthor = allAuthor + ','
        print(allAuthor)
        if allAuthor == '':
            allAuthor = " ——,"
        pp = paper.find('span',attrs={'class':'articlePageRange'}).getText()
        pp = pp[6:]
        doi = paper.find('a',attrs={'class':'ref nowrap abs'}).get('href')
        doi = doi[9:]
        Result = "-" + allAuthor + " 2020, "+ name + ", 20(3): " + pp + ". [-PDF1-](https://sci-hub.ren/" + doi + "), [-PDF2-](https://journals.sagepub.com/doi/pdf/" + doi + ")"

        # print (Result)
        md = open("sj.md",'a')
        md.write(Result +"\n")
        # print(name)
        # print(author)
        # print(pp)
        # print(doi)
        allResult.append(Result)
        md.close()


# Nicholas J. Cox, 2020, Speaking Stata: More ways for rowwise, Stata Joural, 20(2): 481–488. -PDF1-, -PDF2-
# 作者，2020，TItle,Stata Joural, 20(2): 页码， 双pdf地址

def read_html(file):
    with open(file, "r", encoding="utf-8") as f:
        html = f.read()
    return html

def main():
        # md = open("sj.md",'a')
        outerHtmlName = "203.txt"
        html = read_html(outerHtmlName)
        parse_html(html)
        # md.close()

if __name__ == '__main__':
    main()